package crawler

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/jackdanger/collectlinks"
)

// func init() {
// 	url := "http://www.baidu.com/"

// 	queue := make(chan string, 3)

// 	go func() {
// 		queue <- url
// 	}()
// 	for uri := range queue {
// 		go download(uri, queue)
// 		time.Sleep(100 * time.Millisecond)
// 	}
// }

// func download(url string) {
// 	client := &http.Client{}
// 	req, _ := http.NewRequest("GET", url, nil)
// 	// 自定义Header
// 	req.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")

// 	resp, err := client.Do(req)
// 	if err != nil {
// 		fmt.Println("http get error", err)
// 		return
// 	}
// 	//函数结束后关闭相关链接
// 	defer resp.Body.Close()

// 	// body, err := ioutil.ReadAll(resp.Body)
// 	// if err != nil {
// 	// 	fmt.Println("read error", err)
// 	// 	return
// 	// }
// 	links := collectlinks.All(resp.Body)
// 	for _, link := range links {
// 		fmt.Println("parse url", link)
// 	}
// }

var visited = make(map[string]bool)

func download(url string, queue chan string) {
	visited[url] = true
	println("url size:", len(visited))
	client := &http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	// 自定义Header
	req.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")

	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("http get error", err)
		return
	}
	//函数结束后关闭相关链接
	defer resp.Body.Close()

	links := collectlinks.All(resp.Body)
	for _, link := range links {
		absolute := urlJoin(link, url)
		if url != " " {
			println("parse url:", absolute)
			if !visited[absolute] {
				go func() {
					queue <- absolute
				}()
			}

		}
	}
}

func urlJoin(href, base string) string {
	uri, err := url.Parse(href)
	if err != nil {
		return " "
	}
	baseUrl, err := url.Parse(base)
	if err != nil {
		return " "
	}
	return baseUrl.ResolveReference(uri).String()
}

func downloadImage(url string, path string) {
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println("访问图片出错:" + url)
	}

	_data, _err2 := ioutil.ReadAll(resp.Body)
	if _err2 != nil {
		panic(_err2)
	}
	//保存到本地
	ioutil.WriteFile(path, _data, 0644)
}
